{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import joblib\n",
    "import warnings\n",
    "import os\n",
    "import collections\n",
    "from itertools import zip_longest\n",
    "\n",
    "from scipy.optimize import curve_fit\n",
    "import GPy\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from utils import cl_curve_smooth, curve_derivative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_01 = joblib.load('./test_HA/test_HA_01.lz4')\n",
    "test_02 = joblib.load('./test_HA/test_HA_02.lz4')\n",
    "test_03 = joblib.load('./test_HA/test_HA_03.lz4')\n",
    "test_04 = joblib.load('./test_HA/test_HA_04.lz4')\n",
    "test_05 = joblib.load('./test_HA/test_HA_05.lz4')\n",
    "\n",
    "train_01 = joblib.load('./train_HA/train_HA_01.lz4')\n",
    "train_02 = joblib.load('./train_HA/train_HA_02.lz4')\n",
    "train_03 = joblib.load('./train_HA/train_HA_03.lz4')\n",
    "\n",
    "train_02 = train_02[train_02['CLI']>45]\n",
    "train_03 = train_03[train_03['CLI']<140]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def CL_mean_initial(df, apply_col, seg_col, up_thred = 30, down_thred= 0, show_anomaly=True):\n",
    "    '''\n",
    "    描述：\n",
    "        统计每一段时间的某一列均值->改成中位数，更稳\n",
    "    '''\n",
    "    data = df[apply_col]\n",
    "    seg_cl = df[seg_col].unique()\n",
    "    seg_cl.sort()\n",
    "    \n",
    "    \n",
    "    # 初始化mean_list\n",
    "    mean_list = []\n",
    "    for cl in seg_cl:\n",
    "        seg_mean = data[df[seg_col] == cl].median()\n",
    "        mean_list.append(seg_mean)\n",
    "    if show_anomaly:\n",
    "        for i in range(len(mean_list)):\n",
    "            idx_mean = mean_list[i]\n",
    "            if (idx_mean>up_thred) or (idx_mean<down_thred):\n",
    "                mean_list[i] = -1\n",
    "    return seg_cl, mean_list\n",
    "\n",
    "def CL_percent_delta_initial(df, apply_col, seg_col, up_box=90, down_box=10, up_thred=10, down_thred=0, show_anomaly = False):\n",
    "    '''\n",
    "    描述：\n",
    "        初始化统计量 每一段时间的某一列变化范围(delta)，\n",
    "        注意：如果需要进一步使用cl_curve_smooth函数平滑曲线的话， show_anomaly应该设置为False\n",
    "    '''\n",
    "    data = df[apply_col]\n",
    "    seg_cl = df[seg_col].unique()\n",
    "    seg_cl.sort()\n",
    "    \n",
    "    # 初始化delta_list\n",
    "    delta_list = []\n",
    "    for cl in seg_cl:\n",
    "        tmp = data[df[seg_col] == cl]\n",
    "        up_bound = np.percentile(tmp, up_box)\n",
    "        down_bound = np.percentile(tmp, down_box)\n",
    "        cl_delta = up_bound - down_bound\n",
    "        delta_list.append(cl_delta)\n",
    "    if show_anomaly:\n",
    "        for i in range(len(delta_list)):\n",
    "            idx_delta = delta_list[i]\n",
    "            if (idx_delta>up_thred) or (idx_delta<down_thred):\n",
    "                delta_list[i] = -1\n",
    "\n",
    "    return seg_cl, delta_list\n",
    "\n",
    "def gen_feature(df, has_label=False, feature_list=['PCA_T2__mean', 'PCA_T2__mean_diff', 'PCA_T2__delta', 'PCA_T2__delta_diff']):\n",
    "    \n",
    "    # mean， mean_diff\n",
    "    cl, mean_list = CL_mean_initial(df, 'PCA_T2', 'CLI', 30, 0, show_anomaly=False)\n",
    "    cl, mean_list, _ = cl_curve_smooth(mean_list, cl, up_thred=30, down_thred=0, confidence=0.5, fit_type='moving_robust_avg')\n",
    "    \n",
    "    cl, mean_list, fit_func = cl_curve_smooth(mean_list, cl, up_thred=30, down_thred=0, confidence=0.0, fit_type='scipy_curve_fit', return_cureve_func=True)\n",
    "    mean_diff_list = curve_derivative(fit_func, cl, 1)\n",
    "    \n",
    "    cl, delta_list = CL_percent_delta_initial(df, 'PCA_T2', 'CLI', up_box=95, down_box=5, show_anomaly=False)\n",
    "    cl, delta_list, _ = cl_curve_smooth(delta_list, cl, up_thred=30, down_thred=0, confidence=0.5, fit_type='moving_robust_avg')\n",
    "    delta_list = pd.Series(delta_list).rolling(window=4, center=False, min_periods=1).mean().tolist()\n",
    "    cl, delta_list, fit_func_dict = cl_curve_smooth(delta_list, cl, up_thred=10, down_thred=0, confidence=0.0, fit_type='scipy_curve_fit', return_cureve_func=True)\n",
    "    delta_diff_list = curve_derivative(fit_func_dict, cl, 1)\n",
    "    \n",
    "    if has_label:\n",
    "        rulr = df.groupby(by='CLI')['RULR'].mean().tolist()\n",
    "        data = np.array([mean_list, mean_diff_list, delta_list, delta_diff_list, cl, rulr]).T\n",
    "        result = pd.DataFrame(data=data, columns=feature_list + ['CL', 'RULR'])\n",
    "    else:\n",
    "        data = np.array([mean_list, mean_diff_list, delta_list, delta_diff_list, cl]).T\n",
    "        result = pd.DataFrame(data=data, columns= feature_list + ['CL'])\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_list = [train_01, train_02, train_03]\n",
    "test_list = [test_01, test_02, test_03, test_04, test_05]\n",
    "\n",
    "# train\n",
    "if not os.path.exists('./train_HA_feature'):\n",
    "    os.mkdir('./train_HA_feature')\n",
    "for i, train in enumerate(train_list):\n",
    "    tmp = gen_feature(df=train, has_label=True)\n",
    "    joblib.dump(tmp, './train_HA_feature/train_0%d.lz4'%(i+1), compress='lz4')\n",
    "    \n",
    "# test    \n",
    "if not os.path.exists('./test_HA_feature'):\n",
    "    os.mkdir('./test_HA_feature')\n",
    "for i, test in enumerate(test_list):\n",
    "    tmp = gen_feature(df=test, has_label=False)\n",
    "    joblib.dump(tmp, './test_HA_feature/test_0%d.lz4'%(i+1), compress='lz4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
